package com.xieh.webmagic.magic;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


/**
 * @author 谢辉
 * @Classname MyWebMagic
 * @Description TODO
 * @Date 2020/11/20 22:07
 *
 * http://webmagic.io/apidocs/
 *
 * http://webmagic.io/docs/zh/
 */
public class MyWebMagic implements PageProcessor {

    private Site site = Site.me().setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36 OPR/72.0.3815.207")
            .setRetryTimes(3)
            .setSleepTime(1000);

    // /html/gndy/jddy/20160320/50523.html
    private String regxPath = "/html/gndy/\\w{4}/\\d{8}/\\d{5}.html";

    private Pattern pattern = Pattern.compile(regxPath);

    @Override
    public void process(Page page) {

        System.out.println("page.getUrl:" + page.getUrl().get());
        System.out.println(page.getUrl().get().equals("https://www.dytt8.net/index.htm"));

        // 如果首页，执行该逻辑
        if(page.getUrl().toString().equals("https://www.dytt8.net/index.htm")){
            System.out.println("是首页....");
            // 得到页面的源代码
            Html html = page.getHtml();

            // 注意这里匹配到的是@href的内容。例如/html/gndy/jddy/20160320/50523.html
            //String movieXpath = "//div[@class='co_content2']/ul/a/@href";

            // 注意这里匹配到的是html代码！！！！例如：<a href="/html/gndy/jddy/20160320/50523.html">...........</a>
            String movieXpath = "//div[@class='co_content2']/ul/a[@href]";

            Selectable selectXpath = html.xpath(movieXpath);
            List<String> allLinks = selectXpath.all();

            System.out.println("匹配到的元素："+allLinks.toString());

            for(int i = 1; i < allLinks.size(); i++){
                Matcher matcher = pattern.matcher(allLinks.get(i));
                if(matcher.find()){
                    System.out.println("matcher.group:" + matcher.group());

                    // 放list连接
                    // page.addTargetRequests(allLinks);

                    // 自动访问放入的链接地址，并继续执行process方法里的逻辑
                    page.addTargetRequest(matcher.group());
                }
            }

            System.out.println("getTargetRequests:" + page.getTargetRequests().toString());


        // 如果不是首页执行的逻辑
        }else{

            System.out.println("不是首页....");
            Html html = page.getHtml();
            String movieNameXpath = "//title/text()";
            String movieLinkXpath = "//a[starts-with(@href,'ftp')]/text()";

            Selectable movieNameSelectable = html.xpath(movieNameXpath);
            Selectable movieLinkSelectable = html.xpath(movieLinkXpath);

            // 表示都有匹配,结果是一个
            if(movieNameSelectable.match() && movieLinkSelectable.match()){
                String movieName = movieNameSelectable.get();
                String movieLink = movieLinkSelectable.get();
                System.out.println("电影名称：" + movieName + "\t链接:" + movieLink);
                // 记录信息，使用MyPipeline保存到文件中
                page.putField("name",movieName);
                page.putField("link",movieLink);
            }


        }




    }

    @Override
    public Site getSite() {
        return this.site;
    }

    public static void main(String[] args) {
        Spider.create(new MyWebMagic())
                .addUrl("https://www.dytt8.net/index.htm")
                .addPipeline(new MyPipeline())
                .thread(1)
                .run();
    }
}
